<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - layers.h</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font><font color='#0000FF'>#ifndef</font> DLIB_DNn_LAYERS_H_
<font color='#0000FF'>#define</font> DLIB_DNn_LAYERS_H_

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='layers_abstract.h.html'>layers_abstract.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tensor.h.html'>tensor.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='core.h.html'>core.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>iostream<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>string<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../rand.h.html'>../rand.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../string.h.html'>../string.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tensor_tools.h.html'>tensor_tools.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../vectorstream.h.html'>../vectorstream.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='utilities.h.html'>utilities.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>sstream<font color='#5555FF'>&gt;</font>


<font color='#0000FF'>namespace</font> dlib
<b>{</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _num_filters,
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc,
        <font color='#0000FF'><u>int</u></font> _stride_y,
        <font color='#0000FF'><u>int</u></font> _stride_x,
        <font color='#0000FF'><u>int</u></font> _padding_y <font color='#5555FF'>=</font> _stride_y<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nr<font color='#5555FF'>/</font><font color='#979000'>2</font>,
        <font color='#0000FF'><u>int</u></font> _padding_x <font color='#5555FF'>=</font> _stride_x<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nc<font color='#5555FF'>/</font><font color='#979000'>2</font>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='con_'></a>con_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_num_filters <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of filters must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows in a filter must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns in a filter must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>&lt;</font> _nr, "<font color='#CC0000'>The padding must be smaller than the filter size.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>&lt;</font> _nc, "<font color='#CC0000'>The padding must be smaller than the filter size.</font>"<font face='Lucida Console'>)</font>;

        <b><a name='con_'></a>con_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> : 
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            padding_y_<font face='Lucida Console'>(</font>_padding_y<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>_padding_x<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>long</u></font> <b><a name='num_filters'></a>num_filters</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _num_filters; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nr'></a>nr</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nr; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nc'></a>nc</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nc; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_y'></a>stride_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_y; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_x'></a>stride_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_x; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_y'></a>padding_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_y_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_x'></a>padding_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_x_; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>
            point p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>
            point p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <b><a name='con_'></a>con_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : 
            params<font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>,
            filters<font face='Lucida Console'>(</font>item.filters<font face='Lucida Console'>)</font>,
            biases<font face='Lucida Console'>(</font>item.biases<font face='Lucida Console'>)</font>,
            learning_rate_multiplier<font face='Lucida Console'>(</font>item.learning_rate_multiplier<font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font>item.weight_decay_multiplier<font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier<font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier<font face='Lucida Console'>)</font>,
            padding_y_<font face='Lucida Console'>(</font>item.padding_y_<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>item.padding_x_<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// this-&gt;conv is non-copyable and basically stateless, so we have to write our
</font>            <font color='#009900'>// own copy to avoid trying to copy it and getting an error.
</font>        <b>}</b>

        con_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            <font color='#009900'>// this-&gt;conv is non-copyable and basically stateless, so we have to write our
</font>            <font color='#009900'>// own copy to avoid trying to copy it and getting an error.
</font>            params <font color='#5555FF'>=</font> item.params;
            filters <font color='#5555FF'>=</font> item.filters;
            biases <font color='#5555FF'>=</font> item.biases;
            padding_y_ <font color='#5555FF'>=</font> item.padding_y_;
            padding_x_ <font color='#5555FF'>=</font> item.padding_x_;
            learning_rate_multiplier <font color='#5555FF'>=</font> item.learning_rate_multiplier;
            weight_decay_multiplier <font color='#5555FF'>=</font> item.weight_decay_multiplier;
            bias_learning_rate_multiplier <font color='#5555FF'>=</font> item.bias_learning_rate_multiplier;
            bias_weight_decay_multiplier <font color='#5555FF'>=</font> item.bias_weight_decay_multiplier;
            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>long</u></font> num_inputs <font color='#5555FF'>=</font> _nr<font color='#5555FF'>*</font>_nc<font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> num_outputs <font color='#5555FF'>=</font> _num_filters;
            <font color='#009900'>// allocate params for the filters and also for the filter bias values.
</font>            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_inputs<font color='#5555FF'>*</font>_num_filters <font color='#5555FF'>+</font> _num_filters<font face='Lucida Console'>)</font>;

            dlib::rand <font color='#BB00BB'>rnd</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>rand</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>randomize_parameters</font><font face='Lucida Console'>(</font>params, num_inputs<font color='#5555FF'>+</font>num_outputs, rnd<font face='Lucida Console'>)</font>;

            filters <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>_num_filters, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, _nr, _nc<font face='Lucida Console'>)</font>;
            biases <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,_num_filters<font face='Lucida Console'>)</font>;

            <font color='#009900'>// set the initial bias values to zero
</font>            <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params,filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>conv</font><font face='Lucida Console'>(</font>output,
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>,
                _stride_y,
                _stride_x,
                padding_y_,
                padding_x_
                <font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,output,<font color='#979000'>1</font>,<font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params,filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            conv.<font color='#BB00BB'>get_gradient_for_data</font> <font face='Lucida Console'>(</font>gradient_input, <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// no point computing the parameter gradients if they won't be used.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>learning_rate_multiplier <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> filt <font color='#5555FF'>=</font> <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params_grad,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                conv.<font color='#BB00BB'>get_gradient_for_filters</font> <font face='Lucida Console'>(</font>gradient_input, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filt<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params_grad, filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>assign_conv_bias_gradient</font><font face='Lucida Console'>(</font>b, gradient_input<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>con_4</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_num_filters, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_x, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_y_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_x_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.filters, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.biases, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>con_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> num_filters;
            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#0000FF'><u>int</u></font> stride_y;
            <font color='#0000FF'><u>int</u></font> stride_x;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>con_4</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>num_filters, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_y, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_x, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_y_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_x_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.filters, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.biases, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_y_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_y found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_x_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_x found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>num_filters <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _num_filters<font face='Lucida Console'>)</font> 
                <b>{</b>
                    std::ostringstream sout;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Wrong num_filters found while deserializing dlib::con_</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::endl;
                    sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>expected </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> _num_filters <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> but found </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_filters <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::endl;
                    <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>stride_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _stride_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_y found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>stride_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _stride_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_x found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::con_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>con\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>num_filters=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_num_filters
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;con</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> num_filters='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_num_filters<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/con&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        resizable_tensor params;
        alias_tensor filters, biases;

        tt::tensor_conv conv;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;

        <font color='#009900'>// These are here only because older versions of con (which you might encounter
</font>        <font color='#009900'>// serialized to disk) used different padding settings.
</font>        <font color='#0000FF'><u>int</u></font> padding_y_;
        <font color='#0000FF'><u>int</u></font> padding_x_;

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> num_filters,
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> con <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>con_<font color='#5555FF'>&lt;</font>num_filters,nr,nc,stride_y,stride_x<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc,
        <font color='#0000FF'><u>int</u></font> _stride_y,
        <font color='#0000FF'><u>int</u></font> _stride_x,
        <font color='#0000FF'><u>int</u></font> _padding_y <font color='#5555FF'>=</font> _stride_y<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nr<font color='#5555FF'>/</font><font color='#979000'>2</font>,
        <font color='#0000FF'><u>int</u></font> _padding_x <font color='#5555FF'>=</font> _stride_x<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nc<font color='#5555FF'>/</font><font color='#979000'>2</font>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='max_pool_'></a>max_pool_</b>
    <b>{</b>
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>&lt;</font> _nr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nc<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>&lt;</font> _nc<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>public</font>:


        <b><a name='max_pool_'></a>max_pool_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> :
            padding_y_<font face='Lucida Console'>(</font>_padding_y<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>_padding_x<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>long</u></font> <b><a name='nr'></a>nr</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nr; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nc'></a>nc</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nc; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_y'></a>stride_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_y; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_x'></a>stride_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_x; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_y'></a>padding_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_y_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_x'></a>padding_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_x_; <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>
            point p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>
            point p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <b><a name='max_pool_'></a>max_pool_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>  :
            padding_y_<font face='Lucida Console'>(</font>item.padding_y_<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>item.padding_x_<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// this-&gt;mp is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>        <b>}</b>

        max_pool_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            padding_y_ <font color='#5555FF'>=</font> item.padding_y_;
            padding_x_ <font color='#5555FF'>=</font> item.padding_x_;

            <font color='#009900'>// this-&gt;mp is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            mp.<font color='#BB00BB'>setup_max_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>mp</font><font face='Lucida Console'>(</font>output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            mp.<font color='#BB00BB'>setup_max_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            mp.<font color='#BB00BB'>get_gradient</font><font face='Lucida Console'>(</font>gradient_input, computed_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>max_pool_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_x, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_y_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_x_, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>max_pool_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#0000FF'><u>int</u></font> stride_y;
            <font color='#0000FF'><u>int</u></font> stride_x;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>max_pool_2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_y, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_x, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_y_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_x_, in<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::max_pool_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_y_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_y found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_x_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_x found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_y found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_x found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>max_pool (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;max_pool</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:


        tt::pooling mp;
        resizable_tensor params;

        <font color='#0000FF'><u>int</u></font> padding_y_;
        <font color='#0000FF'><u>int</u></font> padding_x_;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> max_pool <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>max_pool_<font color='#5555FF'>&lt;</font>nr,nc,stride_y,stride_x<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> max_pool_everything <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>max_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,<font color='#979000'>0</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc,
        <font color='#0000FF'><u>int</u></font> _stride_y,
        <font color='#0000FF'><u>int</u></font> _stride_x,
        <font color='#0000FF'><u>int</u></font> _padding_y <font color='#5555FF'>=</font> _stride_y<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nr<font color='#5555FF'>/</font><font color='#979000'>2</font>,
        <font color='#0000FF'><u>int</u></font> _padding_x <font color='#5555FF'>=</font> _stride_x<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nc<font color='#5555FF'>/</font><font color='#979000'>2</font>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='avg_pool_'></a>avg_pool_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>&lt;</font> _nr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nc<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>&lt;</font> _nc<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;

        <b><a name='avg_pool_'></a>avg_pool_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> :
            padding_y_<font face='Lucida Console'>(</font>_padding_y<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>_padding_x<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>long</u></font> <b><a name='nr'></a>nr</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nr; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nc'></a>nc</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nc; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_y'></a>stride_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_y; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_x'></a>stride_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_x; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_y'></a>padding_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_y_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_x'></a>padding_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_x_; <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>
            point p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>
            point p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <b><a name='avg_pool_'></a>avg_pool_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>  :
            padding_y_<font face='Lucida Console'>(</font>item.padding_y_<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>item.padding_x_<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// this-&gt;ap is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>        <b>}</b>

        avg_pool_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            padding_y_ <font color='#5555FF'>=</font> item.padding_y_;
            padding_x_ <font color='#5555FF'>=</font> item.padding_x_;

            <font color='#009900'>// this-&gt;ap is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            ap.<font color='#BB00BB'>setup_avg_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>ap</font><font face='Lucida Console'>(</font>output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            ap.<font color='#BB00BB'>setup_avg_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            ap.<font color='#BB00BB'>get_gradient</font><font face='Lucida Console'>(</font>gradient_input, computed_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>avg_pool_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_x, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_y_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_x_, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>avg_pool_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#0000FF'><u>int</u></font> stride_y;
            <font color='#0000FF'><u>int</u></font> stride_x;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>avg_pool_2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_y, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_x, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_y_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_x_, in<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::avg_pool_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_y_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_y found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_x_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_x found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_y found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_x found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>avg_pool (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;avg_pool</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>
    <font color='#0000FF'>private</font>:

        tt::pooling ap;
        resizable_tensor params;

        <font color='#0000FF'><u>int</u></font> padding_y_;
        <font color='#0000FF'><u>int</u></font> padding_x_;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> avg_pool <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>avg_pool_<font color='#5555FF'>&lt;</font>nr,nc,stride_y,stride_x<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> avg_pool_everything <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>avg_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,<font color='#979000'>0</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>enum</font> <b><a name='layer_mode'></a>layer_mode</b>
    <b>{</b>
        CONV_MODE <font color='#5555FF'>=</font> <font color='#979000'>0</font>,
        FC_MODE <font color='#5555FF'>=</font> <font color='#979000'>1</font>
    <b>}</b>;

    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> DEFAULT_BATCH_NORM_EPS <font color='#5555FF'>=</font> <font color='#979000'>0.0001</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        layer_mode mode
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='bn_'></a>bn_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='bn_'></a>bn_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> window_size,
            <font color='#0000FF'><u>double</u></font> eps_ <font color='#5555FF'>=</font> DEFAULT_BATCH_NORM_EPS
        <font face='Lucida Console'>)</font> : 
            num_updates<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>, 
            running_stats_window_size<font face='Lucida Console'>(</font>window_size<font face='Lucida Console'>)</font>,
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            eps<font face='Lucida Console'>(</font>eps_<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_size <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The batch normalization running stats window size can't be 0.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <b><a name='bn_'></a>bn_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> : bn_<font face='Lucida Console'>(</font><font color='#979000'>100</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        layer_mode <b><a name='get_mode'></a>get_mode</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> mode; <b>}</b>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='get_running_stats_window_size'></a>get_running_stats_window_size</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> running_stats_window_size; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_running_stats_window_size'></a>set_running_stats_window_size</b> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> new_window_size <font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>new_window_size <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The batch normalization running stats window size can't be 0.</font>"<font face='Lucida Console'>)</font>;
            running_stats_window_size <font color='#5555FF'>=</font> new_window_size; 
        <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_eps'></a>get_eps</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> eps; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            beta <font color='#5555FF'>=</font> gamma;

            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>beta.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            running_means.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            running_variances.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            running_means <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            running_variances <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            num_updates <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> decay <font color='#5555FF'>=</font> <font color='#979000'>1.0</font> <font color='#5555FF'>-</font> num_updates<font color='#5555FF'>/</font><font face='Lucida Console'>(</font>num_updates<font color='#5555FF'>+</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_updates;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>num_updates <font color='#5555FF'>&gt;</font> running_stats_window_size<font face='Lucida Console'>)</font>
                    num_updates <font color='#5555FF'>=</font> running_stats_window_size;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>eps, output, means, invstds, decay, running_means, running_variances, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font> 
                    tt::<font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>eps, output, means, invstds, decay, running_means, running_variances, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#009900'>// we are running in testing mode so we just linearly scale the input tensor.
</font>            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>batch_normalize_inference</font><font face='Lucida Console'>(</font>eps, output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b, running_means, running_variances<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    tt::<font color='#BB00BB'>batch_normalize_conv_inference</font><font face='Lucida Console'>(</font>eps, output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b, running_means, running_variances<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> g_grad <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params_grad, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b_grad <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params_grad, gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>eps, gradient_input, means, invstds, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g_grad, b_grad <font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>eps, gradient_input, means, invstds, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g_grad, b_grad <font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> bn_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> CONV_MODE<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>bn_con2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font> <font color='#009900'>// if FC_MODE
</font>                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>bn_fc2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gamma, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.beta, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.means, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.invstds, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.running_means, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.running_variances, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_updates, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.running_stats_window_size, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.eps, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>bn_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> CONV_MODE<font face='Lucida Console'>)</font> 
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_con2</font>"<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::bn_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#009900'>// must be in FC_MODE
</font>            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_fc2</font>"<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::bn_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gamma, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.beta, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.means, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.invstds, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.running_means, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.running_variances, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_updates, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.running_stats_window_size, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.eps, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> bn_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>bn_con  </font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>bn_fc   </font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> running_stats_window_size=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.running_stats_window_size;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> bn_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;bn_con</font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;bn_fc</font>";

            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> running_stats_window_size='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.running_stats_window_size<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";

            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/bn_con&gt;\n</font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/bn_fc&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> affine_;

        resizable_tensor params;
        alias_tensor gamma, beta;
        resizable_tensor means, running_means;
        resizable_tensor invstds, running_variances;
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_updates;
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> running_stats_window_size;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> eps;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> bn_con <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>bn_<font color='#5555FF'>&lt;</font>CONV_MODE<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> bn_fc <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>bn_<font color='#5555FF'>&lt;</font>FC_MODE<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>class</font> <b><a name='visitor_bn_running_stats_window_size'></a>visitor_bn_running_stats_window_size</b>
        <b>{</b>
        <font color='#0000FF'>public</font>:

            <b><a name='visitor_bn_running_stats_window_size'></a>visitor_bn_running_stats_window_size</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> new_window_size_<font face='Lucida Console'>)</font> : new_window_size<font face='Lucida Console'>(</font>new_window_size_<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'><u>void</u></font> <b><a name='set_window_size'></a>set_window_size</b><font face='Lucida Console'>(</font>T<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#009900'>// ignore other layer detail types
</font>            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font> layer_mode mode <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'><u>void</u></font> <b><a name='set_window_size'></a>set_window_size</b><font face='Lucida Console'>(</font>bn_<font color='#5555FF'>&lt;</font>mode<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> l<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
            <b>{</b>
                l.<font color='#BB00BB'>set_running_stats_window_size</font><font face='Lucida Console'>(</font>new_window_size<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> input_layer_type<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'><u>void</u></font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> , input_layer_type<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>  <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#009900'>// ignore other layers
</font>            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T, <font color='#0000FF'>typename</font> U, <font color='#0000FF'>typename</font> E<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'><u>void</u></font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> , add_layer<font color='#5555FF'>&lt;</font>T,U,E<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> l<font face='Lucida Console'>)</font>  <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#BB00BB'>set_window_size</font><font face='Lucida Console'>(</font>l.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

        <font color='#0000FF'>private</font>:

            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> new_window_size;
        <b>}</b>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> net_type<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='set_all_bn_running_stats_window_sizes'></a>set_all_bn_running_stats_window_sizes</b> <font face='Lucida Console'>(</font>
        net_type<font color='#5555FF'>&amp;</font> net,
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> new_window_size
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>visit_layers</font><font face='Lucida Console'>(</font>net, impl::<font color='#BB00BB'>visitor_bn_running_stats_window_size</font><font face='Lucida Console'>(</font>new_window_size<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>enum</font> <b><a name='fc_bias_mode'></a>fc_bias_mode</b>
    <b>{</b>
        FC_HAS_BIAS <font color='#5555FF'>=</font> <font color='#979000'>0</font>,
        FC_NO_BIAS <font color='#5555FF'>=</font> <font color='#979000'>1</font>
    <b>}</b>;

    <font color='#0000FF'>struct</font> <b><a name='num_fc_outputs'></a>num_fc_outputs</b>
    <b>{</b>
        <b><a name='num_fc_outputs'></a>num_fc_outputs</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> n<font face='Lucida Console'>)</font> : num_outputs<font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font> <b>{</b><b>}</b>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs_,
        fc_bias_mode bias_mode
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='fc_'></a>fc_</b>
    <b>{</b>
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>num_outputs_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of outputs from a fc_ layer must be &gt; 0</font>"<font face='Lucida Console'>)</font>;

    <font color='#0000FF'>public</font>:
        <b><a name='fc_'></a>fc_</b><font face='Lucida Console'>(</font>num_fc_outputs o<font face='Lucida Console'>)</font> : num_outputs<font face='Lucida Console'>(</font>o.num_outputs<font face='Lucida Console'>)</font>, num_inputs<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <b><a name='fc_'></a>fc_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> : fc_<font face='Lucida Console'>(</font>num_fc_outputs<font face='Lucida Console'>(</font>num_outputs_<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='get_num_outputs'></a>get_num_outputs</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> num_outputs; <b>}</b>

        fc_bias_mode <b><a name='get_bias_mode'></a>get_bias_mode</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> bias_mode; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            num_inputs <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS<font face='Lucida Console'>)</font>
                params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_inputs<font color='#5555FF'>+</font><font color='#979000'>1</font>, num_outputs<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_inputs, num_outputs<font face='Lucida Console'>)</font>;

            dlib::rand <font color='#BB00BB'>rnd</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>rand</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>randomize_parameters</font><font face='Lucida Console'>(</font>params, num_inputs<font color='#5555FF'>+</font>num_outputs, rnd<font face='Lucida Console'>)</font>;

            weights <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>num_inputs, num_outputs<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS<font face='Lucida Console'>)</font>
            <b>{</b>
                biases <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,num_outputs<font face='Lucida Console'>)</font>;
                <font color='#009900'>// set the initial bias values to zero
</font>                <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params,weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, num_outputs<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> w <font color='#5555FF'>=</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,output, <font color='#979000'>1</font>,sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>false</font>, w,<font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,output,<font color='#979000'>1</font>,b<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// no point computing the parameter gradients if they won't be used.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>learning_rate_multiplier <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// compute the gradient of the weight parameters.  
</font>                <font color='#0000FF'>auto</font> pw <font color='#5555FF'>=</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params_grad, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,pw, <font color='#979000'>1</font>,sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>true</font>, gradient_input,<font color='#979000'>false</font><font face='Lucida Console'>)</font>;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#009900'>// compute the gradient of the bias parameters.  
</font>                    <font color='#0000FF'>auto</font> pb <font color='#5555FF'>=</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params_grad, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    tt::<font color='#BB00BB'>assign_bias_gradient</font><font face='Lucida Console'>(</font>pb, gradient_input<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>

            <font color='#009900'>// compute the gradient for the data
</font>            <font color='#0000FF'>auto</font> w <font color='#5555FF'>=</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>1</font>,gradient_input,<font color='#979000'>false</font>, w,<font color='#979000'>true</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_instance <b><a name='get_weights'></a>get_weights</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_const_instance <b><a name='get_weights'></a>get_weights</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_instance <b><a name='get_biases'></a>get_biases</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS, "<font color='#CC0000'>This fc_ layer doesn't have a bias vector </font>"
                "<font color='#CC0000'>to be retrieved, as per template parameter 'bias_mode'.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_const_instance <b><a name='get_biases'></a>get_biases</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS, "<font color='#CC0000'>This fc_ layer doesn't have a bias vector </font>"
                "<font color='#CC0000'>to be retrieved, as per template parameter 'bias_mode'.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> fc_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>fc_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_outputs, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_inputs, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weights, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.biases, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font><font face='Lucida Console'>)</font>bias_mode, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>fc_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>fc_2</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::fc_.</font>"<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_outputs, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_inputs, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weights, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.biases, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>int</u></font> bmode <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>bmode, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>fc_bias_mode<font face='Lucida Console'>)</font>bmode<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong fc_bias_mode found while deserializing dlib::fc_</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> fc_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>fc\t (</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>num_outputs=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>fc_no_bias (</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>num_outputs=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            <b>}</b>
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> fc_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>FC_HAS_BIAS<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;fc</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> num_outputs='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/fc&gt;\n</font>";
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;fc_no_bias</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> num_outputs='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/fc_no_bias&gt;\n</font>";
            <b>}</b>
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs;
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_inputs;
        resizable_tensor params;
        alias_tensor weights, biases;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> fc <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>fc_<font color='#5555FF'>&lt;</font>num_outputs,FC_HAS_BIAS<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> fc_no_bias <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>fc_<font color='#5555FF'>&lt;</font>num_outputs,FC_NO_BIAS<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='dropout_'></a>dropout_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='dropout_'></a>dropout_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> drop_rate_ <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>
        <font face='Lucida Console'>)</font> :
            drop_rate<font face='Lucida Console'>(</font>drop_rate_<font face='Lucida Console'>)</font>,
            rnd<font face='Lucida Console'>(</font>std::rand<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> drop_rate <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> drop_rate <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#009900'>// We have to add a copy constructor and assignment operator because the rnd object
</font>        <font color='#009900'>// is non-copyable.
</font>        <b><a name='dropout_'></a>dropout_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : drop_rate<font face='Lucida Console'>(</font>item.drop_rate<font face='Lucida Console'>)</font>, mask<font face='Lucida Console'>(</font>item.mask<font face='Lucida Console'>)</font>, rnd<font face='Lucida Console'>(</font>std::rand<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        dropout_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            drop_rate <font color='#5555FF'>=</font> item.drop_rate;
            mask <font color='#5555FF'>=</font> item.mask;
            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_drop_rate'></a>get_drop_rate</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> drop_rate; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// create a random mask and use it to filter the data
</font>            mask.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>input<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>mask<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>mask, drop_rate<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output, input, mask<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, data_grad, mask, gradient_input<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, data_grad, mask, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>dropout_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.drop_rate, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.mask, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>dropout_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>dropout_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::dropout_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.drop_rate, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.mask, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>dropout\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>drop_rate=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.drop_rate
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;dropout</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> drop_rate='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.drop_rate<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>float</u></font> drop_rate;
        resizable_tensor mask;

        tt::tensor_rand rnd;
        resizable_tensor params; <font color='#009900'>// unused
</font>    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> dropout <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>dropout_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='multiply_'></a>multiply_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='multiply_'></a>multiply_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> val_ <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>
        <font face='Lucida Console'>)</font> :
            val<font face='Lucida Console'>(</font>val_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <b><a name='multiply_'></a>multiply_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : val<font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>item.get_drop_rate<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_multiply_value'></a>get_multiply_value</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> val; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>output, input, val<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>data_grad, gradient_input, val<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>data_grad, data_grad, gradient_input, <font color='#979000'>1</font>, val<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> multiply_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>multiply_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.val, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>multiply_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>dropout_</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Since we can build a multiply_ from a dropout_ we check if that's what
</font>                <font color='#009900'>// is in the stream and if so then just convert it right here.
</font>                unserialize <font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                dropout_ temp;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>temp, sin<font face='Lucida Console'>)</font>;
                item <font color='#5555FF'>=</font> temp;
                <font color='#0000FF'>return</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>multiply_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::multiply_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.val, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> multiply_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>multiply (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>val=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.val
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> multiply_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;multiply</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> val='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.val<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>
    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>float</u></font> val;
        resizable_tensor params; <font color='#009900'>// unused
</font>    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> multiply <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>multiply_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='affine_'></a>affine_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='affine_'></a>affine_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> : mode<font face='Lucida Console'>(</font>FC_MODE<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <b><a name='affine_'></a>affine_</b><font face='Lucida Console'>(</font>
            layer_mode mode_
        <font face='Lucida Console'>)</font> : mode<font face='Lucida Console'>(</font>mode_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            layer_mode bnmode
            <font color='#5555FF'>&gt;</font>
        <b><a name='affine_'></a>affine_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> bn_<font color='#5555FF'>&lt;</font>bnmode<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            gamma <font color='#5555FF'>=</font> item.gamma;
            beta <font color='#5555FF'>=</font> item.beta;
            mode <font color='#5555FF'>=</font> bnmode;

            params.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            
            resizable_tensor <font color='#BB00BB'>temp</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> sg <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>temp,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> sb <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>temp,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            g <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>sg<font face='Lucida Console'>)</font>, <font color='#979000'>1.0f</font><font color='#5555FF'>/</font><font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.running_variances<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>item.<font color='#BB00BB'>get_eps</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            b <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>sb<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>g<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.running_means<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        layer_mode <b><a name='get_mode'></a>get_mode</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> mode; <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            beta <font color='#5555FF'>=</font> gamma;

            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>beta.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>output, input, g, b<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>affine_transform_conv</font><font face='Lucida Console'>(</font>output, input, g, b<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// We are computing the gradient of dot(gradient_input, computed_output*g + b)
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    tt::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> empty_params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> empty_params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> affine_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>affine_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gamma, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.beta, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font><font face='Lucida Console'>)</font>item.mode, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>affine_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_con2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Since we can build an affine_ from a bn_ we check if that's what is in
</font>                <font color='#009900'>// the stream and if so then just convert it right here.
</font>                unserialize <font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                bn_<font color='#5555FF'>&lt;</font>CONV_MODE<font color='#5555FF'>&gt;</font> temp;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>temp, sin<font face='Lucida Console'>)</font>;
                item <font color='#5555FF'>=</font> temp;
                <font color='#0000FF'>return</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_fc2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Since we can build an affine_ from a bn_ we check if that's what is in
</font>                <font color='#009900'>// the stream and if so then just convert it right here.
</font>                unserialize <font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                bn_<font color='#5555FF'>&lt;</font>FC_MODE<font color='#5555FF'>&gt;</font> temp;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>temp, sin<font face='Lucida Console'>)</font>;
                item <font color='#5555FF'>=</font> temp;
                <font color='#0000FF'>return</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>affine_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::affine_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gamma, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.beta, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>int</u></font> mode;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>mode, in<font face='Lucida Console'>)</font>;
            item.mode <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>layer_mode<font face='Lucida Console'>)</font>mode;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> affine_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>affine</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> affine_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;affine</font>";
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> mode='conv'</font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> mode='fc'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/affine&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params, empty_params; 
        alias_tensor gamma, beta;
        layer_mode mode;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> affine <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>affine_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_prev_'></a>add_prev_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>::id;

        <b><a name='add_prev_'></a>add_prev_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t1 <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t2 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>output, t1, t2<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// The gradient just flows backwards to the two layers that forward() added
</font>            <font color='#009900'>// together.
</font>            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_prev_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>add_prev_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_prev_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>add_prev_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::add_prev_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_prev_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>add_prev</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_prev_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;add_prev tag='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> add_prev <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>add_prev_<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev1  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag1, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev2  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag2, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev3  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag3, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev4  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag4, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev5  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag5, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev6  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag6, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev7  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag7, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev8  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag8, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev9  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag9, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev10 <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag10, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>using</font> add_prev1_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev2_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev3_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag3<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev4_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag4<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev5_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag5<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev6_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag6<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev7_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag7<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev8_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag8<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev9_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag9<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev10_ <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag10<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='relu_'></a>relu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='relu_'></a>relu_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>relu</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>relu_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> relu_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>relu_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>relu_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>relu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::relu_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> relu_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>relu</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> relu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;relu/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> relu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>relu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='prelu_'></a>prelu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='prelu_'></a>prelu_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> initial_param_value_ <font color='#5555FF'>=</font> <font color='#979000'>0.25</font>
        <font face='Lucida Console'>)</font> : initial_param_value<font face='Lucida Console'>(</font>initial_param_value_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_initial_param_value'></a>get_initial_param_value</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> initial_param_value; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            params <font color='#5555FF'>=</font> initial_param_value;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, 
            resizable_tensor<font color='#5555FF'>&amp;</font> data_output
        <font face='Lucida Console'>)</font>
        <b>{</b>
            data_output.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>prelu</font><font face='Lucida Console'>(</font>data_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, params<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> params_grad
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>prelu_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                gradient_input, params, params_grad<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> prelu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>prelu_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.initial_param_value, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>prelu_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>prelu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::prelu_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.initial_param_value, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> prelu_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>prelu\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>initial_param_value=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.initial_param_value
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> prelu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;prelu initial_param_value='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.initial_param_value<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/prelu&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        <font color='#0000FF'><u>float</u></font> initial_param_value;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> prelu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>prelu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='sig_'></a>sig_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='sig_'></a>sig_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>sigmoid_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> sig_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>sig_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>sig_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>sig_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::sig_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> sig_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>sig</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> sig_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;sig/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> sig <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>sig_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='htan_'></a>htan_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='htan_'></a>htan_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>inline</font> point <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> point <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>tanh_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> htan_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>htan_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>htan_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>htan_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::htan_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> htan_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>htan</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> htan_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;htan/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> htan <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>htan_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='softmax_'></a>softmax_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='softmax_'></a>softmax_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>softmax_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> softmax_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>softmax_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>softmax_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>softmax_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::softmax_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> softmax_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>softmax</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> softmax_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;softmax/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> softmax <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>softmax_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b>, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font>... <b><a name='TAG_TYPES'></a>TAG_TYPES</b><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='concat_helper_impl'></a>concat_helper_impl</b><b>{</b>

            constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><font color='#0000FF'>return</font> <font color='#979000'>1</font> <font color='#5555FF'>+</font> concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;<b>}</b>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font> ? "<font color='#CC0000'>,</font>" : "<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='resize_out'></a>resize_out</b><font face='Lucida Console'>(</font>resizable_tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>long</u></font> sum_k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>resize_out</font><font face='Lucida Console'>(</font>out, sub, sum_k <font color='#5555FF'>+</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='concat'></a>concat</b><font face='Lucida Console'>(</font>tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>out, k_offset, t, <font color='#979000'>0</font>, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                k_offset <font color='#5555FF'>+</font><font color='#5555FF'>=</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>concat</font><font face='Lucida Console'>(</font>out, sub, k_offset<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='split'></a>split</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>t, <font color='#979000'>0</font>, input, k_offset, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                k_offset <font color='#5555FF'>+</font><font color='#5555FF'>=</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>input, sub, k_offset<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='concat_helper_impl'></a>concat_helper_impl</b><font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><b>{</b>
            constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><font color='#0000FF'>return</font> <font color='#979000'>1</font>;<b>}</b>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font> 
            <b>{</b> 
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id;
            <b>}</b>

            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='resize_out'></a>resize_out</b><font face='Lucida Console'>(</font>resizable_tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>long</u></font> sum_k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                out.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>t.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sum_k, t.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='concat'></a>concat</b><font face='Lucida Console'>(</font>tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>out, k_offset, t, <font color='#979000'>0</font>, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='split'></a>split</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font>t, <font color='#979000'>0</font>, input, k_offset, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
    <b>}</b>
    <font color='#009900'>// concat layer
</font>    <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font>... <b><a name='TAG_TYPES'></a>TAG_TYPES</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='concat_'></a>concat_</b>
    <b>{</b>
        <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font> <b>{</b> impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;<b>}</b>;

    <font color='#0000FF'>public</font>:
        constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><font color='#0000FF'>return</font> impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;<b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// do nothing
</font>        <b>}</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// the total depth of result is the sum of depths from all tags
</font>            impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>resize_out</font><font face='Lucida Console'>(</font>output, sub, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// copy output from each tag into different part result
</font>            impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>concat</font><font face='Lucida Console'>(</font>output, sub, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// Gradient is split into parts for each tag layer
</font>            impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>gradient_input, sub, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        point <b><a name='map_input_to_output'></a>map_input_to_output</b><font face='Lucida Console'>(</font>point p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;
        point <b><a name='map_output_to_input'></a>map_output_to_input</b><font face='Lucida Console'>(</font>point p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> concat_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>concat_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>count, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>concat_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>concat_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::concat_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count_tags;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>count_tags, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>count_tags <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Invalid count of tags </font>"<font color='#5555FF'>+</font> std::<font color='#BB00BB'>to_string</font><font face='Lucida Console'>(</font>count_tags<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font>"<font color='#CC0000'>, expecting </font>" <font color='#5555FF'>+</font>
                                          std::<font color='#BB00BB'>to_string</font><font face='Lucida Console'>(</font><font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font>
                                                  "<font color='#CC0000'> found while deserializing dlib::concat_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> concat_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>concat\t (</font>";
            <font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> concat_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;concat tags='</font>";
            <font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params; <font color='#009900'>// unused
</font>    <b>}</b>;


    <font color='#009900'>// concat layer definitions
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat2 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG3'></a>TAG3</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat3 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2, TAG3<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG3'></a>TAG3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG4'></a>TAG4</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat4 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2, TAG3, TAG4<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG3'></a>TAG3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG4'></a>TAG4</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG5'></a>TAG5</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat5 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2, TAG3, TAG4, TAG5<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#009900'>// inception layer will use tags internally. If user will use tags too, some conflicts
</font>    <font color='#009900'>// possible to exclude them, here are new tags specially for inceptions
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag0  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>0</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag1  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>1</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag2  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag3  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag4  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>4</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag5  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>5</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#009900'>// skip to inception input
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> iskip  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> itag0, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#009900'>// here are some templates to be used for creating inception layer groups
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception2 <font color='#5555FF'>=</font> concat2<font color='#5555FF'>&lt;</font>itag1, itag2, itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font> itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B3'></a>B3</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception3 <font color='#5555FF'>=</font> concat3<font color='#5555FF'>&lt;</font>itag1, itag2, itag3, itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag3<font color='#5555FF'>&lt;</font>B3<font color='#5555FF'>&lt;</font>  itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B3'></a>B3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B4'></a>B4</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception4 <font color='#5555FF'>=</font> concat4<font color='#5555FF'>&lt;</font>itag1, itag2, itag3, itag4,
                itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag3<font color='#5555FF'>&lt;</font>B3<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font>  itag4<font color='#5555FF'>&lt;</font>B4<font color='#5555FF'>&lt;</font>  itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B3'></a>B3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B4'></a>B4</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B5'></a>B5</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception5 <font color='#5555FF'>=</font> concat5<font color='#5555FF'>&lt;</font>itag1, itag2, itag3, itag4, itag5,
                itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag3<font color='#5555FF'>&lt;</font>B3<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font>  itag4<font color='#5555FF'>&lt;</font>B4<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font>  itag5<font color='#5555FF'>&lt;</font>B5<font color='#5555FF'>&lt;</font>  itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_DNn_LAYERS_H_
</font>


</pre></body></html>